# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import re
import csv
from pymongo import MongoClient


requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告


"""
搜狗微信-只显示100页
https://weixin.sogou.com/
1.获取搜索内容
2.根据首页中的链接解析真实地址
"""


class SogouWeiXin():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        # requests.utils.dict_from_cookiejar(html.cookies)  # 获取cookies转dict
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            # 'User-Agent': ua.random,
            # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Connection': 'keep-alive',
            'Cookie': 'ABTEST=4|1588723796|v1; IPLOC=CN2103; SUID=DAEA113C7D29990A000000005EB20054; SUID=DAEA113C3121A00A000000005EB20054; weixinIndexVisited=1; SUV=00236AB23C11EADA5EB20054EAACD111; ppinf=5|1588724058|1589933658|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZToxNDE6JUU1JTk1JTg2JUU3JTk0JUE4JUVGJUJDJThDJUU2JUIwJTkxJUU3JTk0JUE4JUU1JTg3JTgwJUU2JUIwJUI0JUU1JTk5JUE4JUU3JUJCJUI0JUU0JUJGJUFFJTI4JUU0JUI4JTkzJUU1JUFFJUI2JTI5JUU4JTgwJTgxJUU5JTgzJUFEJUU1JUE0JUI0fGNydDoxMDoxNTg4NzI0MDU4fHJlZm5pY2s6MTQxOiVFNSU5NSU4NiVFNyU5NCVBOCVFRiVCQyU4QyVFNiVCMCU5MSVFNyU5NCVBOCVFNSU4NyU4MCVFNiVCMCVCNCVFNSU5OSVBOCVFNyVCQiVCNCVFNCVCRiVBRSUyOCVFNCVCOCU5MyVFNSVBRSVCNiUyOSVFOCU4MCU4MSVFOSU4MyVBRCVFNSVBNCVCNHx1c2VyaWQ6NDQ6bzl0Mmx1T3ZzRGM4X1k5T3RhcWpzb2F3V09mUUB3ZWl4aW4uc29odS5jb218; pprdig=oLQpeC-8CPpucYg2pN0_LbxtFm7IqgFRVFwxFIdD3JsDB5RIJb7vR3v6YcxrEUr4wJsGg5OXzGtf_1TnsJoTeftGE--xHHpzj-RuYPCutNK7v9B9Kqj470t-K2sAJdfvx8nYQBRDdF0TU8OeqMLm3-eEgQmUSsIWQMDKhCOk78A; sgid=07-45715645-AV6yAVoibzpZ1VpHblmP0rYs; ppmdig=1588724058000000d2d9b20567b2797feb62490c9ade4902; JSESSIONID=aaa2UjnoXlky562db3Rgx; PHPSESSID=mqdkvccjkhn2n2a1809tqj63u2; SNUID=E5AA517C3F3AE4B0245943C54062734B',
            'DNT': '1',
            'Host': 'weixin.sogou.com',
            'Referer': 'https://weixin.sogou.com/weixin?oq=&query=%E5%9B%BA%E5%AE%9A%E8%B5%84%E4%BA%A7%E6%8A%95%E8%B5%84&_sug_type_=1&sut=0&lkt=0%2C0%2C0&s_from=input&ri=0&_sug_=n&type=2&sst0=1588650305092&page=13&ie=utf8&p=40040108&dp=1&w=01015002&dr=1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'same-origin',
            'Sec-Fetch-User': '?1',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36'

        }

        self.headers_child = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Connection': 'keep-alive',
            'Cookie': 'ABTEST=4|1588723796|v1; IPLOC=CN2103; SUID=DAEA113C7D29990A000000005EB20054; SUID=DAEA113C3121A00A000000005EB20054; weixinIndexVisited=1; SUV=00236AB23C11EADA5EB20054EAACD111; ppinf=5|1588724058|1589933658|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZToxNDE6JUU1JTk1JTg2JUU3JTk0JUE4JUVGJUJDJThDJUU2JUIwJTkxJUU3JTk0JUE4JUU1JTg3JTgwJUU2JUIwJUI0JUU1JTk5JUE4JUU3JUJCJUI0JUU0JUJGJUFFJTI4JUU0JUI4JTkzJUU1JUFFJUI2JTI5JUU4JTgwJTgxJUU5JTgzJUFEJUU1JUE0JUI0fGNydDoxMDoxNTg4NzI0MDU4fHJlZm5pY2s6MTQxOiVFNSU5NSU4NiVFNyU5NCVBOCVFRiVCQyU4QyVFNiVCMCU5MSVFNyU5NCVBOCVFNSU4NyU4MCVFNiVCMCVCNCVFNSU5OSVBOCVFNyVCQiVCNCVFNCVCRiVBRSUyOCVFNCVCOCU5MyVFNSVBRSVCNiUyOSVFOCU4MCU4MSVFOSU4MyVBRCVFNSVBNCVCNHx1c2VyaWQ6NDQ6bzl0Mmx1T3ZzRGM4X1k5T3RhcWpzb2F3V09mUUB3ZWl4aW4uc29odS5jb218; pprdig=oLQpeC-8CPpucYg2pN0_LbxtFm7IqgFRVFwxFIdD3JsDB5RIJb7vR3v6YcxrEUr4wJsGg5OXzGtf_1TnsJoTeftGE--xHHpzj-RuYPCutNK7v9B9Kqj470t-K2sAJdfvx8nYQBRDdF0TU8OeqMLm3-eEgQmUSsIWQMDKhCOk78A; sgid=07-45715645-AV6yAVoibzpZ1VpHblmP0rYs; ppmdig=1588724058000000d2d9b20567b2797feb62490c9ade4902; JSESSIONID=aaa2UjnoXlky562db3Rgx; PHPSESSID=mqdkvccjkhn2n2a1809tqj63u2; SNUID=E5AA517C3F3AE4B0245943C54062734B',
            'DNT': '1',
            'Host': 'weixin.sogou.com',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36'
        }
        self.headers_real = {
            'User-Agent': ua.random,
        }

    # 获取列表信息
    def get_contents(self, cursor):
        postData = {
            'query': '固定资产投资 深圳',
            '_sug_type_':'',
            'sut': '5226',
            'lkt': '2,1588673471146,1588673475103',
            's_from': 'input',
            '_sug_': 'n',
            'type': '2',
            'sst0': '1588673475206',
            'page': cursor,#'2',
            'ie': 'utf8',
            'w': '01030402',
            'dr': '1'
        }
        url = f"https://weixin.sogou.com/weixin?query=%E5%9B%BA%E5%AE%9A%E8%B5%84%E4%BA%A7%E6%8A%95%E8%B5%84+%E6%B7%B1%E5%9C%B3&_sug_type_=&s_from=input&_sug_=n&type=2&page={cursor}&ie=utf8"
        time.sleep(2)
        html = HttpUtils.do_request("GET", url, self.headers, "")
        root = etree.HTML(html.text)
        selector = root.xpath('//div[@class="txt-box"]')
        if len(selector) > 0:
            for x in selector:
                dict_data = dict()
                dict_data['title'] = "".join(x.xpath('./h3/a//text()'))
                dict_data['titleFqr'] = x.xpath('./div[@class="s-p"]/a/text()')[0]
                push_time = x.xpath('./div[@class="s-p"]/span//text()')[0]
                push_time = push_time.replace("document.write(timeConvert('", "").replace("'))", "")
                dict_data['titleTime'] = datetime.fromtimestamp(int(push_time)).strftime("%Y-%m-%d")
                dict_data['titleContent'] = "".join(x.xpath('./p[@class="txt-info"]//text()'))
                dict_data['titleUrl'] = x.xpath('./h3/a/@href')[0]
                url_child = dict_data['titleUrl']
                print(f"=============标题：{dict_data['title']}=================")
                if url_child.find("http") < 0:
                    dict_data['titleUrl'] = "https://weixin.sogou.com" + url_child
                # 解析真实地址
                time.sleep(2)
                print("==========解析真实地址============")
                html_child = HttpUtils.do_request("GET", dict_data['titleUrl'], self.headers_child, "")
                url_real = "".join(re.findall(r"url \+\= (.*?);", html_child.text)).replace("'", "").replace("@", "")
                # 获取文章内容
                time.sleep(1)
                print("==========获取文章内容============")
                child_res = HttpUtils.do_request("GET", url_real, self.headers_real, "")
                root_child = etree.HTML(child_res.text)
                dict_data['content'] = "".join(root_child.xpath('//div[@id="js_content"]//text()')).replace("\r", "").replace("\n", "").strip()
                self.insertItem("搜狗微信_固定资产投资深圳", dict_data)
                print(str(dict_data))
                time.sleep(3)
        else:
            print(f"==========验证码,重新获取cookies:{url}============")
            return len(selector)

    def insertItem(self, tableName, data):
        my_set = db[tableName]
        my_set.insert_one(data)


if __name__ == '__main__':
    conn = MongoClient('127.0.0.1', 27017)
    db = conn["Test"]
    sougou = SogouWeiXin()
    for i in range(48, 101):
        print(f"============第【{i}】页=============")
        result = sougou.get_contents(i)
        # 如果返回0，则有验证码
        if result == 0:
            break